package GDoc::Job::Fetch::US::Exec::Treas::FMS::CSROB;

use Moose;


extends 'GDoc::Job::Fetch';

sub setup {
    my $self = shift;
    $self->_baseurl( 'http://fms.treas.gov/annualreport/backissues.html' );
    $self->_basedir( 'http://fms.treas.gov/annualreport/' );
}

sub run { 
    my $self = shift;
    my %data = %{ $self->data };

    if ( $data{link} ) { 
        $self->_fetch_pdf;
    } elsif ( $data{index} ) { 
        $self->_fetch_index;
    } else { 
        $self->logger->info( "Fetching base " . $self->_baseurl );
        my $mech = $self->_mech;
        $mech->get( $self->_baseurl );
        my @links = $mech->find_all_links( url_regex => qr{cs\d{4}/index\.html} );
        
        foreach my $link( @links ) { 
            my $job = GDoc::Job->new( class => 'Fetch::US::Exec::Treas::FMS::CSROB',
                                      queue => $self->fetch_queue,
                                      data  => { index => $self->_basedir . $link->url }
                                    );
            $job->submit;
        }
    }
}

sub _fetch_index { 
    my $self = shift;
    my %data = %{ $self->data };

    my $link = $data{index};
    $self->logger->info( "Fetching index page $link" );

    $link =~ m{^(?<base>.*?/)index\.html$};
    my $basedir = $+{base};

    my $mech = $self->_mech;
    $mech->get( $link );

    my $html = $mech->response->content;
    
    my $secnum = 0;
    while ( $html =~ m{<p>\s?(?<title>[\w\s\'-]+)</p>.*?<a href="[\w\./]*?(?<url>\w+.pdf)">}gsi ) { 
        my ( $title, $url ) = @+{'title', 'url'};
        $title =~ s/\n|\r//g;
        
        my $job = GDoc::Job->new( class => 'Fetch::US::Exec::Treas::FMS::CSROB',
                                  queue => $self->fetch_queue,
                                  data  => { link     => $basedir . $url,
                                             desc     => $title,
                                             secnum   => ++$secnum,
                                           }
                                );
        $job->submit;
    }
}


sub _fetch_pdf { 
    my $self = shift;
    my %data = %{ $self->data };

    my ( $link, $desc, $secnum ) = @data{ 'link', 'desc', 'secnum' };

    $link =~ m{cs(?<year>\d{4})};
    my $year = $+{year};

    my $ident = sprintf( 'FMS-CSROB-%04d-%02d', $year, $secnum );
    my $filename = $ident . '.pdf';
    my $key = 'us/exec/treas/fms/' . $filename;

    $self->logger->info( "Storing $link --> $key" );

    my $mech = $self->_mech;
    $mech->get( $link );

    $self->store_pdf( key       => $key,
                      data      => $mech->response->content,
                      filename  => $filename,
                      meta      => { description => $desc,
                                     identifier  => $ident
                                   }
                    );
}

    

1;
        
